NexusPi Git Node
Commit 4e5d4ee8ad4bd4d37b3cbf6b64663c5908d8f97a
Parents : 59784a3
Author : James L <jrl290@gmail.com>
Date : 2026-02-27T17:51:20-05:00
v1.0.8: Fix heap exhaustion from unbounded random_blobs
Root cause: Python Reticulum trims random_blobs per destination entry
(MAX_RANDOM_BLOBS=64 in-memory, PERSIST_RANDOM_BLOBS=32 on disk).
The C++ firmware had these constants defined but NEVER enforced them,
causing unbounded growth. With 21 paths x 60+ blobs x ~90 bytes each,
the destination table alone consumed ~57KB of the ESP32 324KB heap.
Fixes:
- Trim random_blobs after insert (matching Python behavior)
- Trim random_blobs on deserialization from flash
- Trim random_blobs to PERSIST_RANDOM_BLOBS on serialization
- Enforce _path_table_maxpersist when writing path table (was declared
but never used - write_path_table saved everything)
- Reduce MCU constants: MAX_RANDOM_BLOBS 64->16, PERSIST_RANDOM_BLOBS 32->8
- Reduce path_table_maxsize 128->24, maxpersist 32->12
- Add memory diagnostic after path table load
- Trim loaded paths to maxsize on startup via cull_path_table()
Results: destination_table 21KB->5.8KB, free heap 63K(22%)->156K(49%)
Changes
4 files changed, 95 insertions(+), 13 deletions(-)
Diff
diff --git a/RNode_Firmware.ino b/RNode_Firmware.ino
index e8d5255..7870053 100755
--- a/RNode_Firmware.ino
+++ b/RNode_Firmware.ino
@@ -648,12 +648,11 @@ void setup() {
// ── Boundary Mode: Load config and optionally set up WiFi + TCP ──
HEAD("Boundary Mode: Initializing...", RNS::LOG_TRACE);
- // With TLSF pool moved to PSRAM we have plenty of room.
- // 128 path entries supports ~15-20 devices comfortably.
- // cull_path_table() is patched to evict backbone paths first, preserving
- // local (LoRa / local-TCP) paths needed for inbound message delivery.
- RNS::Transport::path_table_maxsize(128);
- RNS::Transport::path_table_maxpersist(32);
+ // ESP32 has only ~324KB heap. Each path entry with random_blobs costs
+ // ~200-500 bytes. Keep tables small to avoid heap exhaustion.
+ // cull_path_table() evicts backbone paths first, preserving local ones.
+ RNS::Transport::path_table_maxsize(24);
+ RNS::Transport::path_table_maxpersist(12);
boundary_load_config();
// Start WiFi if enabled
diff --git a/lib/microReticulum/src/Transport.cpp b/lib/microReticulum/src/Transport.cpp
index 8d80273..c72c44c 100755
--- a/lib/microReticulum/src/Transport.cpp
+++ b/lib/microReticulum/src/Transport.cpp
@@ -2256,6 +2256,26 @@ static bool is_backbone_interface(const Interface& iface) {
random_blobs.insert(random_blob);
+ // Trim random_blobs to prevent unbounded memory growth
+ // (matching Python: random_blobs = random_blobs[-MAX_RANDOM_BLOBS:])
+ if (random_blobs.size() > MAX_RANDOM_BLOBS) {
+ // Keep only the MAX_RANDOM_BLOBS blobs with the highest
+ // timestamps (bytes 5-9 big-endian). Extract, sort by
+ // timestamp desc, keep top N, rebuild set.
+ std::vector<Bytes> blob_vec(random_blobs.begin(), random_blobs.end());
+ std::sort(blob_vec.begin(), blob_vec.end(), [](const Bytes& a, const Bytes& b) {
+ // Sort descending by timestamp (bytes 5-9)
+ uint64_t ts_a = OS::from_bytes_big_endian(a.data() + 5, 5);
+ uint64_t ts_b = OS::from_bytes_big_endian(b.data() + 5, 5);
+ return ts_a > ts_b;
+ });
+ random_blobs.clear();
+ for (size_t i = 0; i < MAX_RANDOM_BLOBS && i < blob_vec.size(); i++) {
+ random_blobs.insert(blob_vec[i]);
+ }
+ TRACEF("Trimmed random_blobs to %d entries for %s", random_blobs.size(), packet.destination_hash().toHex().c_str());
+ }
+
if ((Reticulum::transport_enabled() || Transport::from_local_client(packet)) && packet.context() != Type::Packet::PATH_RESPONSE) {
// Insert announce into announce table for retransmission
@@ -3993,6 +4013,21 @@ TRACEF("Transport::start: buffer size %d bytes", Persistence::_buffer.size());
for (const auto& destination_hash : invalid_paths) {
_destination_table.erase(destination_hash);
}
+
+ // Enforce maxsize on loaded paths (trim oldest if over limit)
+ if (_destination_table.size() > _path_table_maxsize) {
+ DEBUGF("Transport::start: trimming loaded path table from %d to %d entries", _destination_table.size(), _path_table_maxsize);
+ cull_path_table();
+ }
+
+ // Memory diagnostic after path table load
+ size_t total_blobs = 0;
+ for (const auto& [hash, entry] : _destination_table) {
+ total_blobs += entry._random_blobs.size();
+ }
+ DEBUGF("Transport::start: path table: %d entries, %d total random_blobs (est. %d bytes)",
+ _destination_table.size(), total_blobs, total_blobs * 90);
+
return true;
}
else {
@@ -4044,6 +4079,24 @@ TRACEF("Transport::start: buffer size %d bytes", Persistence::_buffer.size());
double save_start = OS::time();
DEBUGF("Saving %d path table entries to storage...", _destination_table.size());
+ // Enforce maxpersist: create a trimmed copy for serialization
+ // keeping only the most recently used entries (by timestamp)
+ std::map<Bytes, DestinationEntry> persist_table;
+ if (_destination_table.size() <= _path_table_maxpersist) {
+ persist_table = _destination_table;
+ }
+ else {
+ // Sort by timestamp descending, keep only maxpersist entries
+ std::vector<std::pair<Bytes, DestinationEntry>> sorted_entries(_destination_table.begin(), _destination_table.end());
+ std::sort(sorted_entries.begin(), sorted_entries.end(), [](const std::pair<Bytes, DestinationEntry>& a, const std::pair<Bytes, DestinationEntry>& b) {
+ return a.second._timestamp > b.second._timestamp;
+ });
+ for (size_t i = 0; i < _path_table_maxpersist && i < sorted_entries.size(); i++) {
+ persist_table.insert(sorted_entries[i]);
+ }
+ DEBUGF("Trimmed path table from %d to %d entries for persistence", _destination_table.size(), persist_table.size());
+ }
+
/*p
serialised_destinations = []
for destination_hash in Transport.destination_table:
@@ -4087,7 +4140,7 @@ TRACEF("Transport::start: buffer size %d bytes", Persistence::_buffer.size());
#if CUSTOM
{
- Persistence::_document.set(_destination_table);
+ Persistence::_document.set(persist_table);
TRACEF("Transport::write_path_table: doc size %d bytes", Persistence::_document.memoryUsage());
//size_t size = 8192;
@@ -4141,7 +4194,7 @@ TRACE("Transport::write_path_table: buffer size " + std::to_string(Persistence::
TRACE("Transport::write_path_table: failed to serialize");
}
#else // CUSTOM
- uint32_t crc = Persistence::crc(_destination_table);
+ uint32_t crc = Persistence::crc(persist_table);
if (_destination_table_crc > 0 && crc == _destination_table_crc) {
TRACE("Transport::write_path_table: no change detected, skipping write");
}
@@ -4149,8 +4202,8 @@ TRACE("Transport::write_path_table: buffer size " + std::to_string(Persistence::
TRACE("Transport::write_path_table: change detected, writing...");
char destination_table_path[Type::Reticulum::FILEPATH_MAXSIZE];
snprintf(destination_table_path, Type::Reticulum::FILEPATH_MAXSIZE, "%s/destination_table", Reticulum::_storagepath);
- if (Persistence::serialize(_destination_table, destination_table_path, _destination_table_crc) > 0) {
- TRACEF("Transport::write_path_table: wrote %d entries, %d bytes", _destination_table.size(), Persistence::_buffer.size());
+ if (Persistence::serialize(persist_table, destination_table_path, _destination_table_crc) > 0) {
+ TRACEF("Transport::write_path_table: wrote %d entries, %d bytes", persist_table.size(), Persistence::_buffer.size());
success = true;
}
}
diff --git a/lib/microReticulum/src/Type.h b/lib/microReticulum/src/Type.h
index 4fbf066..9c022e8 100755
--- a/lib/microReticulum/src/Type.h
+++ b/lib/microReticulum/src/Type.h
@@ -430,8 +430,8 @@ namespace RNS { namespace Type {
//static const uint16_t MAX_RECEIPTS = 1024; // Maximum number of receipts to keep track of
static const uint16_t MAX_RECEIPTS = 20; // Maximum number of receipts to keep track of
static const uint8_t MAX_RATE_TIMESTAMPS = 16; // Maximum number of announce timestamps to keep per destination
- static const uint8_t PERSIST_RANDOM_BLOBS = 32; // Maximum number of random blobs per destination to persist to disk
- static const uint8_t MAX_RANDOM_BLOBS = 64; // Maximum number of random blobs per destination to keep in memory
+ static const uint8_t PERSIST_RANDOM_BLOBS = 8; // Maximum number of random blobs per destination to persist to disk (reduced for MCU memory)
+ static const uint8_t MAX_RANDOM_BLOBS = 16; // Maximum number of random blobs per destination to keep in memory (reduced for MCU memory)
// CBA MCU
//static const uint32_t DESTINATION_TIMEOUT = 60*60*24*7; // Destination table entries are removed if unused for one week
diff --git a/lib/microReticulum/src/Utilities/Persistence.h b/lib/microReticulum/src/Utilities/Persistence.h
index cdf0c89..c8b3f7c 100755
--- a/lib/microReticulum/src/Utilities/Persistence.h
+++ b/lib/microReticulum/src/Utilities/Persistence.h
@@ -4,6 +4,7 @@
// then they MUST be included BEFORE this header is included.
#include "Transport.h"
#include "Type.h"
+#include "Utilities/OS.h"
#include <ArduinoJson.h>
@@ -278,7 +279,23 @@ namespace ArduinoJson {
dst["received_from"] = src._received_from;
dst["announce_hops"] = src._hops;
dst["expires"] = src._expires;
- dst["random_blobs"] = src._random_blobs;
+ // Trim random_blobs to PERSIST_RANDOM_BLOBS before writing to disk
+ if (src._random_blobs.size() > RNS::Type::Transport::PERSIST_RANDOM_BLOBS) {
+ std::vector<RNS::Bytes> blob_vec(src._random_blobs.begin(), src._random_blobs.end());
+ std::sort(blob_vec.begin(), blob_vec.end(), [](const RNS::Bytes& a, const RNS::Bytes& b) {
+ uint64_t ts_a = RNS::Utilities::OS::from_bytes_big_endian(a.data() + 5, 5);
+ uint64_t ts_b = RNS::Utilities::OS::from_bytes_big_endian(b.data() + 5, 5);
+ return ts_a > ts_b;
+ });
+ std::set<RNS::Bytes> trimmed;
+ for (size_t i = 0; i < RNS::Type::Transport::PERSIST_RANDOM_BLOBS && i < blob_vec.size(); i++) {
+ trimmed.insert(blob_vec[i]);
+ }
+ dst["random_blobs"] = trimmed;
+ }
+ else {
+ dst["random_blobs"] = src._random_blobs;
+ }
/*
//dst["interface_hash"] = src._receiving_interface;
if (src._receiving_interface) {
@@ -320,6 +337,19 @@ namespace ArduinoJson {
dst._hops = src["announce_hops"];
dst._expires = src["expires"];
dst._random_blobs = src["random_blobs"].as<std::set<RNS::Bytes>>();
+ // Trim random_blobs loaded from flash to MAX_RANDOM_BLOBS
+ if (dst._random_blobs.size() > RNS::Type::Transport::MAX_RANDOM_BLOBS) {
+ std::vector<RNS::Bytes> blob_vec(dst._random_blobs.begin(), dst._random_blobs.end());
+ std::sort(blob_vec.begin(), blob_vec.end(), [](const RNS::Bytes& a, const RNS::Bytes& b) {
+ uint64_t ts_a = RNS::Utilities::OS::from_bytes_big_endian(a.data() + 5, 5);
+ uint64_t ts_b = RNS::Utilities::OS::from_bytes_big_endian(b.data() + 5, 5);
+ return ts_a > ts_b;
+ });
+ dst._random_blobs.clear();
+ for (size_t i = 0; i < RNS::Type::Transport::MAX_RANDOM_BLOBS && i < blob_vec.size(); i++) {
+ dst._random_blobs.insert(blob_vec[i]);
+ }
+ }
/*
//dst._receiving_interface = src["interface_hash"];
RNS::Bytes interface_hash = src["interface_hash"];
Served by rngit 1.4.2 - Generated in 0.03s